import requests
import re
import lxml.html
import os

# url = "https://www.qidian.com/all"
etree = lxml.html.etree


class Spider:
    def get_obj(self, url):
        try:
            r = requests.get(url)
            r.raise_for_status()
            e_obj = etree.HTML(r.text)
            return e_obj
        except:
            return "error"

    def first_request(self, url):
        e_obj = self.get_obj(url)
        bk_list = e_obj.xpath('//div[@class="book-mid-info"]/h4/a/text()')
        href_list = e_obj.xpath('//div[@class="book-mid-info"]/h4/a/@href')
        for bk, href in zip(bk_list, href_list):
            if not os.path.exists(bk):
                os.mkdir(bk)
            href = 'http:' + href
            # print(href)
            self.two_request(bk, href)

    def two_request(self, bk, href):
        e_obj = self.get_obj(href)
        title = e_obj.xpath('//ul[@class="cf"]/li/a/text()')
        print(title)



url = "https://www.qidian.com/all"
spider = Spider()
spider.first_request(url)
